//#include "StdAfx.h"
#include "BayesianFilterSerializer.h"


BayesianFilterSerializer::BayesianFilterSerializer(BayesianFilter& filter) 
: filter(filter)
{
}

BayesianFilterSerializer::~BayesianFilterSerializer(void)
{
}

void BayesianFilterSerializer::WriteHeader(ofstream& f, bool data, __int32 id)
{
	__int32 tmp = (__int32) BAYES_MAGIC_NUMBER;	// Magic number
	f.write((char*)&tmp,sizeof(__int32));		
	if ( data )	tmp = 0;						// File type = data
	else tmp = 0xFFFFFFFF;						// File type = matrix
	f.write((char*)&tmp,sizeof(__int32));		
	tmp = (__int32) SERIALIZER_VERSION_FIELD;	// Serializer Version
	f.write((char*)&tmp,sizeof(__int32));
	tmp = (__int32) time(NULL);					// Creation Timestamp
	f.write((char*)&tmp,sizeof(__int32));
	tmp = id;									// File ID
	f.write((char*)&tmp,sizeof(__int32));
	tmp = 0;									// File status 0 = incomplete 1 = complete
	f.write((char*)&tmp,sizeof(__int32));		
	tmp = 0;									// Total fize size
	f.write((char*)&tmp,sizeof(__int32));
	tmp = 0;									// File CRC
	f.write((char*)&tmp,sizeof(__int32));
}

#define FILESIZE_OFFSET		6
#define FILESTATUS_OFFSET	5

void BayesianFilterSerializer::WriteCommit(std::string fileName)
{
	// Para saber el tama~no total tengo que volver a abrirlo...
	// Un bug de Micro$soft? Un feature de C++?
	ifstream dstFileSize;
	dstFileSize.open(fileName.c_str(),ios::ate|ios::binary);
	__int32 size = (__int32) dstFileSize.tellg();
	dstFileSize.close();

	fstream dst;
	dst.open(fileName.c_str(),ios::binary|ios::in|ios::out);
	
	dst.seekp(FILESIZE_OFFSET * sizeof(__int32));
	dst.write((char*)&size,sizeof(__int32));
	size = 0xFFFFFFFF;
	dst.seekp(FILESTATUS_OFFSET * sizeof(__int32));
	dst.write((char*)&size,sizeof(__int32));
	dst.close();	
}

void BayesianFilterSerializer::DumpTagsWords(string dataFile,__int32 fileID)
{
	ofstream dst;
	dst.open(dataFile.c_str(),ios::trunc|ios::binary);

	WriteHeader(dst,true,fileID);
	__int32 tmp = (__int32) filter.tags.size();			// Tag count
	dst.write((char*)&tmp,sizeof(__int32));

	map<string,__BayesianTag*>::iterator iterTags;
	for ( iterTags = filter.tags.begin(); iterTags != filter.tags.end() ; iterTags++ )
	{
		tmp = (__int32) iterTags->second->pos;
		dst.write((char*)&tmp,sizeof(__int32));
		tmp = (__int32) iterTags->second->wordCount;
		dst.write((char*)&tmp,sizeof(__int32));
		tmp = (__int32) iterTags->second->trueCount;
		dst.write((char*)&tmp,sizeof(__int32));
		tmp = (__int32) iterTags->second->falseCount;
		dst.write((char*)&tmp,sizeof(__int32));
		
		tmp = (__int32) iterTags->first.size();
		dst.write((char*)&tmp,sizeof(__int32));
		dst << iterTags->first.c_str();
		for ( int i = 0 ; i < (tmp & 0x3) ; i++ )
			dst << '\0';
	}

	tmp = (__int32) filter.words.size();		// Word count
	dst.write((char*)&tmp,sizeof(__int32));

	// Lista de palabras comprimida con Front Coding
	const char* lastWord = "";
	map<string,__BayesianProbability*>::iterator iterWords;
	for ( iterWords = filter.words.begin() ; iterWords != filter.words.end() ; iterWords++ )
	{
		const char* currentWord = iterWords->first.c_str();
		__int32 repeatCount = 0;

		for ( int i = 0 ; currentWord[i] && lastWord[i] && currentWord[i] == lastWord[i] ; i++ )
			repeatCount++;

		size_t residue = iterWords->first.length() - repeatCount;

		if ( repeatCount < 15 && residue < 15 )
		{
			__int8 tmp = (__int8) ((repeatCount << 4) + residue);
			dst.write((char*) &tmp,1);
		}
		else if ( repeatCount < 15 && residue < (255 + 15) )
		{
			__int8 tmp = (__int8) ((repeatCount << 4) + 0x0F);
			dst.write((char*) &tmp,1);
			tmp = (__int32) (residue - 15);
			dst.write((char*) &tmp,1);
		}
		else if ( repeatCount < (255 + 15) && residue < 15 )
		{
			__int8 tmp = (__int8) (0xF0 + residue);
			dst.write((char*) &tmp,1);
			tmp = repeatCount - 15;
			dst.write((char*) &tmp,1);
		}
		else if ( repeatCount < (255 + 15) && residue < (255 + 15) )
		{
			__int8 tmp = (__int8) 0xFF;
			dst.write((char*) &tmp,1);
			tmp = repeatCount - 15;
			dst.write((char*) &tmp,1);
			tmp = (__int8) (residue - 15);
			dst.write((char*) &tmp,1);
		}
		else
			throw (char*) "BayesianFilterSerializer::Serialize The word is too big!!! (len > 271)";

		dst.write((char*)((size_t) currentWord + repeatCount),(__int32)residue);
		lastWord = currentWord;
	}

	dst.close();
	WriteCommit(dataFile);
}

void BayesianFilterSerializer::DumpMatrix(string matrixFile,__int32 fileID)
{
	ofstream dst;
	dst.open(matrixFile.c_str(),ios::trunc|ios::binary);

	WriteHeader(dst,true,fileID);

	size_t tags = filter.tags.size();
	map<string,__BayesianProbability*>::iterator iter;
	for ( iter = filter.words.begin() ; iter != filter.words.end() ; iter++ )
		dst.write((char*)iter->second,(std::streamsize) (tags*sizeof(__BayesianProbability)));

	dst.close();
	WriteCommit(matrixFile);
}

void BayesianFilterSerializer::Serialize(string dataFile, string matrixFile)
{
	filter.lock.Enter();

	srand((unsigned int)time(NULL));
	__int32 fileID = rand();

	DumpTagsWords(dataFile,fileID);
	DumpMatrix(matrixFile,fileID);

	filter.lock.Release();
}

__int32 BayesianFilterSerializer::CheckHeaders(string fileName,ifstream& dstFile, bool* data)
{
	__int32 size = (__int32) dstFile.tellg();

	dstFile.seekg(0);
	__int32 retval, tmp;
	dstFile.read((char*) &tmp,sizeof(__int32));
	if ( tmp != BAYES_MAGIC_NUMBER )
		throw (char*) "BayesianFilterSerializer::CheckHeaders The file '%s' is not a valid Bayes Filter serialization file.";

	dstFile.read((char*) &tmp,sizeof(__int32));
	if ( tmp != 0x00000000 && tmp != 0xFFFFFFFF )
		throw (char*) "BayesianFilterSerializer::CheckHeader The file '%s' type field is corrupt.";

	*data = (tmp == 0);

	dstFile.read((char*) &tmp,sizeof(__int32));
	if ( tmp != SERIALIZER_VERSION_FIELD )
		throw (char*) "BayesianFilterSerializer::CheckHeader The file '%s' has not been created with this version of the serializer.";

	dstFile.read((char*) &tmp,sizeof(__int32));		// Creation timestamp
	dstFile.read((char*) &retval,sizeof(__int32));	// File ID

	dstFile.read((char*) &tmp,sizeof(__int32));
	if ( tmp != 0xFFFFFFFF )
		throw (char*) "BayesianFilterSerializer::CheckHeader The file '%s' is marked as corrupted.";

	dstFile.read((char*) &tmp,sizeof(__int32));
	if ( size != tmp )
		throw (char*) "BayesianFilterSerializer::CheckHeader The file '%s' size is not valid.";

	dstFile.read((char*) &tmp,sizeof(__int32));		// CRC
	return retval;
}

__int32 BayesianFilterSerializer::RebuildTags(ifstream& data)
{
	__int32 tagCount, strSize;
	data.read((char*)&tagCount,sizeof(__int32));
	if ( tagCount < 0 || tagCount > 65536 )
		throw (char*) "BayesianFilterSerializer::RebuildTags The data file is corrupt: Bad tag count.";

	for ( int i = 0 ; i < tagCount ; i++ )
	{
		__BayesianTag* tmp = new __BayesianTag();
		data.read((char*)&tmp->pos,sizeof(__int32));
		if ( tmp->pos < 0 || tmp->pos >= (unsigned long) tagCount )
			throw (char*) "BayesianFilterSerializer::RebuildTags The data file is corrupt: Bad tag pos.";

		data.read((char*)&tmp->wordCount,sizeof(__int32));
		data.read((char*)&tmp->trueCount,sizeof(__int32));
		data.read((char*)&tmp->falseCount,sizeof(__int32));

		data.read((char*)&strSize,sizeof(__int32));
		if ( strSize <= 0 || strSize > 65536 )
			throw (char*) "BayesianFilterSerializer::RebuildTags The data file is corrupt: Bad tag string size.";

		char* __tagName = new char[strSize+1];
		data.read(__tagName,strSize);
		__tagName[strSize] = 0;
		string tagName(__tagName);

		filter.tags.insert(make_pair(tagName,tmp));

		char crap;
		for ( int j = 0 ; j < (strSize & 0x3) ; j++ )
			data.read(&crap,1);

		if ( data.eof() )
			throw (char*) "BayesianFilterSerializer::RebuildTags The data file is corrupt: Unexpected EOF found reading tags.";
	}

	int* tagCheck = new int [ tagCount ];
	for ( int i = 0 ; i < tagCount ; i++ )
		tagCheck[i] = 0;

	map<string,__BayesianTag*>::iterator iter;
	for ( iter = filter.tags.begin() ; iter != filter.tags.end() ; iter++ )
	{
		__BayesianTag* tmp = iter->second;
		tagCheck[iter->second->pos] = 1;
	}

	for ( int i = 0 ; i < tagCount; i++ )
		if ( tagCheck [ i ] == 0 )
			throw (char*) "BayesianFilterSerializer::RebuildTags The data file is corrupt: Missing tag pos.";

	delete [] tagCheck;
	return tagCount;
}

void BayesianFilterSerializer::RebuildWords(__int32 tagCount,ifstream& data,ifstream& matrix)
{
	__int32 wordCount;
	data.read((char*)&wordCount,sizeof(__int32));
	if ( tagCount < 0 )
		throw (char*) "BayesianFilterSerializer::RebuildWords The data file is corrupt: Bad word count.";

	string lastWord = "";
	char currentWord [ 512 ];
	for ( int i = 0 ; i < wordCount ; i++ )
	{
		if ( data.eof() )
			throw (char*) "BayesianFilterSerializer::RebuildWords The data file is corrupt: Unexpected EOF found reading words.";
		if ( matrix.eof() )
			throw (char*)"BayesianFilterSerializer::RebuildWords The matrix file is corrupt: Unexpected EOF found.";

		__int8 tmp;
		data.read((char*)&tmp,1);
		__int8 repeatCount = ((tmp & 0xF0) >> 4) ;
		__int8 residue     = tmp & 0x0F;

		if ( repeatCount == 15 )
		{
			data.read((char*)&repeatCount,1);
			repeatCount += 15;
		}
		if ( residue == 15 )
		{
			data.read((char*)&residue,1);
			residue += 15;
		}

		data.read(currentWord,residue);
		currentWord[residue] = 0;
		lastWord = lastWord.substr(0,repeatCount) + currentWord;

		__BayesianProbability* row = new __BayesianProbability [ tagCount ];
		matrix.read((char*)row,tagCount*sizeof(__BayesianProbability));

		filter.words.insert(make_pair(lastWord,row));
	}
}

void BayesianFilterSerializer::Deserialize(string dataFile, string matrixFile)
{
	bool ok = false;
	filter.lock.Enter();

	bool data1, data2;
	ifstream data, matrix;
	data.open(dataFile.c_str(),ios::ate|ios::binary);
	matrix.open(matrixFile.c_str(),ios::ate|ios::binary);

	try
	{
		__int32 fileIDdata   = CheckHeaders(dataFile,data,&data1);
		__int32 fileIDmatrix = CheckHeaders(dataFile,matrix,&data2);

		if ( fileIDdata != fileIDmatrix )
			throw (char*) "BayesianFilterSerializer::CheckHeader The files '%s' and '%s' ids do not match.";

		filter.tags.clear();
		filter.words.clear();
		__int32 tagCount = RebuildTags(data);
		RebuildWords(tagCount,data,matrix);

		ok = true;
		throw (char*) "";
	}
	catch ( char* ex )
	{
		data.close();
		matrix.close();
		filter.lock.Release();

		if ( !ok ) throw (char*) ex;
	}
}
